/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.googlecode.crawlerextractor.crawler4j.parser;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

import org.apache.tika.metadata.DublinCore;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.html.HtmlParser;

import com.googlecode.crawlerextractor.crawler4j.crawler.Configurable;
import com.googlecode.crawlerextractor.crawler4j.crawler.CrawlConfig;
import com.googlecode.crawlerextractor.crawler4j.crawler.Page;
import com.googlecode.crawlerextractor.crawler4j.crawler.WebCrawler;
import com.googlecode.crawlerextractor.crawler4j.url.WebURL;
import com.googlecode.crawlerextractor.crawler4j.url.utils.URLCanonicalizer;
import com.googlecode.crawlerextractor.crawler4j.util.Util;

/**
 * @author Yasser Ganjisaffar <lastname at gmail dot com>
 */
public class Parser extends Configurable {

	private final HtmlParser htmlParser;
	private final ParseContext parseContext;

	public Parser(final CrawlConfig config) {
		super(config);
		htmlParser = new HtmlParser();
		parseContext = new ParseContext();
	}

	public boolean parse(final Page page, final WebURL webUrl) {
		return this.parse(page, webUrl, null);
	}

	public boolean parse(final Page page, final WebURL webUrl,
			final WebCrawler webCrawler) {
		String contextURL = webUrl.getUrl();
		if (Util.hasBinaryContent(page.getContentType())) {
			if (!config.isIncludeBinaryContentInCrawling()) {
				return false;
			} else {
				page.setParseData(BinaryParseData.getInstance());
				return true;
			}
		} else if (Util.hasPlainTextContent(page.getContentType())) {
			try {
				final TextParseData parseData = new TextParseData();
				parseData.setTextContent(new String(page.getContentData(), page
						.getContentCharset()));
				page.setParseData(parseData);
				return true;
			} catch (final Exception e) {
				e.printStackTrace();
			}
			return false;
		}

		final Metadata metadata = new Metadata();
		final HtmlContentHandler contentHandler = new HtmlContentHandler();
		InputStream inputStream = null;
		try {
			inputStream = new ByteArrayInputStream(page.getContentData());
			htmlParser.parse(inputStream, contentHandler, metadata,
					parseContext);
		} catch (final Exception e) {
			e.printStackTrace();
		} finally {
			try {
				if (inputStream != null) {
					inputStream.close();
				}
			} catch (final IOException e) {
				e.printStackTrace();
			}
		}

		if (page.getContentCharset() == null) {
			page.setContentCharset(metadata.get("Content-Encoding"));
		}

		final HtmlParseData parseData = new HtmlParseData();
		parseData.setText(contentHandler.getBodyText().trim());
		parseData.setTitle(metadata.get(DublinCore.TITLE));

		final List<WebURL> outgoingUrls = new ArrayList<WebURL>();

		final String baseURL = contentHandler.getBaseUrl();
		if (baseURL != null) {
			contextURL = baseURL;
		}
		boolean shouldParseLink = true;
		if (webCrawler != null) {
			shouldParseLink = webCrawler.shouldScan(webUrl);
		}
		if (shouldParseLink) {
			int urlCount = 0;
			for (final ExtractedUrlAnchorPair urlAnchorPair : contentHandler
					.getOutgoingUrls()) {

				String href = urlAnchorPair.getHref();

				href = href.trim();
				if (href.length() == 0) {
					continue;
				}
				String hrefWithoutProtocol = href.toLowerCase();
				if (href.startsWith("http://")) {
					hrefWithoutProtocol = href.substring(7);
				}
				if (!hrefWithoutProtocol.contains("javascript:")
						&& !hrefWithoutProtocol.contains("@")) {
					final String url = URLCanonicalizer.getCanonicalURL(href,
							contextURL);
					if (url != null) {
						if (webCrawler != null) {
							WebURL curWebUrl = new WebURL(url);
							if (!webCrawler.mustVisit(curWebUrl))
								continue;
						}

						final WebURL webURL = new WebURL();
						webURL.setURL(url);
						webURL.setAnchor(urlAnchorPair.getAnchor());
						outgoingUrls.add(webURL);
						urlCount++;
						if (urlCount > config.getMaxOutgoingLinksToFollow()) {
							break;
						}
					}
				}
			}
		}

		parseData.setOutgoingUrls(outgoingUrls);

		try {
			if (page.getContentCharset() == null) {
				parseData.setHtml(new String(page.getContentData()));
			} else {
				parseData.setHtml(new String(page.getContentData(), page
						.getContentCharset()));
			}
		} catch (final UnsupportedEncodingException e) {
			e.printStackTrace();
			return false;
		}

		page.setParseData(parseData);
		return true;

	}

}
